package org.fujene.memman;

import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import org.fujene.communicate.hash.HashCompareTool;
import org.fujene.communicate.hash.VirtualContentHash;
import org.fujene.constants.Constants;
import org.fujene.logging.LogParser;
import org.fujene.memcon.ReplicateIndexMemory;
import org.fujene.memcon.IndexDisk;
import org.fujene.structure.ContentIDs;
import org.fujene.structure.ContentInfo;
import org.fujene.structure.Queue;
import org.fujene.structure.ExtendTerm;
import org.fujene.structure.IndexQueue;
import org.fujene.structure.InvertIndexData;
import org.fujene.structure.LookupPointer;

public class ReplicateMemoryManager{
	// ネットワークを利用する向けのインデックスシステムを分離
	private static final int DIGLENGTH = Constants.SKIPPTR_INTERVAL;
	private static final int MEM_SKIPPTR = Constants.MEMORY_TYPE_SKIPPTR;
	private static final int MEM_SKIPCONT = Constants.MEMORY_TYPE_SKIPPTRCONT;

	// メモリ・ディスク本体
	private ReplicateIndexMemory memory = null;
	private IndexDisk disk = null;

	public boolean stopFlag = false;

	private int indexcounter = 0;
	private int termcounter = 0;
	private int searchcounter = 0;
	// private Node myTermNode;
	// private Node myContentNode;

	private ContentInfo contentInfo;

	private Lock termLock = new ReentrantLock();
	private Lock invertLock = new ReentrantLock();
	private Lock contentLock = new ReentrantLock();

	public ReplicateMemoryManager(int blockSize, int blockLines, String dirPath, 
			ContentInfo contentInfo, int backupInterval){
		this.disk = new IndexDisk(dirPath);
		this.memory = new ReplicateIndexMemory(blockSize, blockLines, disk, contentInfo,
				backupInterval);
		this.contentInfo = contentInfo;

		LogParser.fine("Memory manager started.");
	}

	public void storeTerm(String term, long ID, int termfreq, int valueType){
		try{
			ExtendTerm newDic = null;
			try{
				termLock.lock();
				// Store term to dictionary, skip if the term exists
				newDic = storeDictionary(term, valueType);
			}finally{
				termLock.unlock();
			}
			// store invert index and IDs
			try{
				invertLock.lock();
				storeInvIndex(newDic.ptr, ID, termfreq, valueType);
			}finally{
				invertLock.unlock();
			}
		}catch(Exception ignored){
			// ignored.printStackTrace();
		}
		if(termcounter++ % 100000 == 0) System.gc();
	}

	// 転置インデックスのポインタを取得
	public ExtendTerm searchTerm(String term, int valueType){
		// 単語検索
		ExtendTerm newTerm = lookupTerm(term, valueType);

		if(newTerm.ptr != null) return newTerm;
		else return null;
	}

	public Queue lookupContent(long ID, int priorIndex){
		int i;
		Queue returnContents = new Queue(contentInfo.getContentNum(), contentInfo.getAppendixNum(), ID);
		LookupPointer priorPtr = null;
		for(i = 0; i < contentInfo.getContentNum(); i++){
			ContentIDs ptrInfo = lookupContentPointer(ID, i);

			if(ptrInfo.currentPtr != null){
				if(i == priorIndex) priorPtr = ptrInfo.currentPtr;
				String content = memory.lookupContents(ptrInfo.currentPtr);
				if(!content.equals("$$$ DELETED")) returnContents.contents[i] = content;
				// else return Constants.EMPTY_QUEUE;
				else returnContents.contents[i] = null;
			}else{
				System.out.println("Cannot found ID " + ID + ", prev: " + ptrInfo.prevPtr + " next: "
						+ ptrInfo.nextPtr + " ID: " + ptrInfo.ID);
				IndexQueue queue = memory.lookupContentsAll(ptrInfo.prevPtr);
				System.out.println("Previous pointer: Next - " + queue.next + " ID - " + queue.ID);
				if(ptrInfo.nextPtr != null){
					queue = memory.lookupContentsAll(ptrInfo.nextPtr);
					System.out.println("Next pointer: Next - " + queue.next + " ID - " + queue.ID);
				}
				return null;
			}
		}

		if(priorPtr != null){
			LookupPointer[] appendixPointers = memory.lookupAppendixPointer(priorPtr);
			for(i = 0; i < appendixPointers.length; i++)
				returnContents.appendices[i] = memory.lookupAppendix(appendixPointers[i]);
		}

		return returnContents;
	}

	public ArrayList<ExtendTerm> getTermList(long hashFrom, long hashTo, int valueType){
		ArrayList<ExtendTerm> terms = new ArrayList<ExtendTerm>();

		// スキップポインタからスタート
		LookupPointer firstPtr = memory.lookupSkipPointer(0);
		LookupPointer nextPtr = firstPtr;
		ExtendTerm queue;
		long hashCode;

		while(nextPtr != null){
			firstPtr = nextPtr;
			queue = memory.lookupDictionary(firstPtr, valueType);
			hashCode = VirtualContentHash.getTermHash(queue.term);
			if(HashCompareTool.existsInMyNode(hashCode)) terms.add(queue);
			nextPtr = memory.lookupDictionaryNext(firstPtr);
		}

		return terms;
	}

	public ArrayList<Queue> getContentList(long hashFrom, long hashTo, int priorIndex){
		ArrayList<Queue> terms = new ArrayList<Queue>();
		int i;
		long hash = 0;
		LookupPointer[] firstPtrs = new LookupPointer[contentInfo.getContentNum()];
		LookupPointer[] nextPtrs = new LookupPointer[contentInfo.getContentNum()];
		LookupPointer[] appendixPtrs = null;
		IndexQueue queue;
		Queue newTerm;

		for(i = 0; i < contentInfo.getContentNum(); i++){
			// スキップポインタからスタート
			firstPtrs[i] = memory.lookupContentsSkipPointer(0, i);
			nextPtrs[i] = firstPtrs[i];
		}

		while(nextPtrs[0] != null){
			newTerm = new Queue(contentInfo.getContentNum(), contentInfo.getAppendixNum(), -1);
			for(i = 0; i < contentInfo.getContentNum(); i++){
				if(nextPtrs[i] == null) break;
				firstPtrs[i] = nextPtrs[i];
				queue = memory.lookupContentsAll(firstPtrs[i]);
				if(i == priorIndex){
					hash = queue.hash;
					newTerm.ID = queue.ID;
				}
				newTerm.contents[i] = queue.value;
				if(newTerm.contents[i] == null) newTerm.contents[i] = new String();
				nextPtrs[i] = memory.lookupNextContentPointer(firstPtrs[i]);
				if(priorIndex == i) appendixPtrs = memory.lookupAppendixPointer(firstPtrs[i]);
			}

			if(appendixPtrs.length != 0){
				for(i = 0; i < contentInfo.getAppendixNum(); i++)
					newTerm.appendices[i] = memory.lookupAppendix(appendixPtrs[i]);
			}
			if(HashCompareTool.existsInMyNode(hash)) terms.add(newTerm);
		}

		return terms;
	}

	public int getCounter(){
		return indexcounter;
	}

	public int getTerms(){
		return termcounter;
	}

	public int getSearchcounter(){
		return searchcounter;
	}

	public void getMemoryList(){
		memory.showMemoryElapsed();
	}

	public void initializeMemory(){
		try{
			memory.readMemory();
		}catch(Exception ignored){}
	}

	public void saveMemory(){
		memory.saveMemories();
	}

	public int getSkipPointerLen(){
		return memory.getSkipPointerLength();
	}

	private ExtendTerm lookupTerm(String query, int valueType){
		searchcounter++;
		// Look at skip pointer first
		int skipPointerLength = memory.getSkipPointerLength();
		LookupPointer dicPtr = null;
		LookupPointer nextPtr = null;
		ExtendTerm storedTerm = null;
		ExtendTerm returnTerm = null;
		int i;
		int skipLevel = 0;
		int skipVal = 1;
		if(skipPointerLength > 1){
			skipLevel = (int)(Math.log(skipPointerLength - 1) / Math.log(DIGLENGTH));
			skipVal = (int)Math.pow(DIGLENGTH, skipLevel);
		}
		for(i = 0; i < skipPointerLength; i += skipVal){
			dicPtr = memory.lookupSkipPointer(i);
			// No pointer
			if(dicPtr == null || dicPtr.equals(LookupPointer.EMPTY)) break;

			storedTerm = memory.lookupDictionary(dicPtr, valueType);
			try{
				storedTerm.skipptr_index = i;
			}catch(Exception e){
				System.out.println("Error at " + dicPtr + " skip pointer is at " + i + "/"
						+ skipPointerLength);
				break;
			}
			int comp = storedTerm.term.compareTo(query);
			if(comp > 0){
				// skip pointer advances -> revert pointer
				skipLevel--;
				if(skipLevel == -1 || i == 0){
					i--;
					break;
				}
				i -= skipVal;
				skipVal = (int)Math.pow(DIGLENGTH, skipLevel);

			}else if(comp == 0) return storedTerm;

			// traverse
			if(i + skipVal >= skipPointerLength){
				do{
					skipLevel--;
					if(skipLevel == -1){
						skipVal = 1;
						break;
					}
					skipVal = (int)Math.pow(DIGLENGTH, skipLevel);
				}while(i + skipVal >= skipPointerLength);
			}
		}
		if(i == -1){
			// prior to the first pointer
			nextPtr = storedTerm.ptr;
			storedTerm = null;
		}else{
			// get skip pointer again
			if(i == skipPointerLength) i--;
			dicPtr = memory.lookupSkipPointer(i);
			storedTerm = memory.lookupDictionary(dicPtr, valueType);
		}

		// find by dictionary order
		if(storedTerm != null){
			LookupPointer prevPtr = storedTerm.ptr;
			ExtendTerm nextTerm = memory.lookupDictionary(storedTerm.next, valueType);
			int count = 0;
			while(nextTerm != null){
				int comp = nextTerm.term.compareTo(query);
				if(comp > 0){
					// Not exists
					boolean addSkipPtr = count >= DIGLENGTH ? true : false;
					returnTerm = new ExtendTerm(query, null, prevPtr, nextTerm.ptr, i, addSkipPtr);
					break;
				}else if(comp == 0){
					nextTerm.prev = prevPtr;
					return nextTerm;
				}else{
					prevPtr = nextTerm.ptr;
					nextTerm = memory.lookupDictionary(nextTerm.next, valueType);
				}
				count++;
			}

			if(returnTerm == null){
				// 最後の単語だった
				boolean addSkipPtr = count >= DIGLENGTH ? true : false;
				returnTerm = new ExtendTerm(query, null, prevPtr, null, i, addSkipPtr);
			}
		}else{
			// 一番先頭(i = -1) or 最初の単語(i = 0)
			returnTerm = new ExtendTerm(query, null, null, nextPtr, i, false);
		}

		return returnTerm;
	}

	private void makeSkipPtrForTerm(ExtendTerm term, int type){
		int index = term.skipptr_index;

		if(index == -1){
			// 最初の部分書き換え
			memory.storeSkipPointer(0, type, term.ptr, true, MEM_SKIPPTR);
			refreshSkipPtrDict();
		}else{
			if(term.needSkipPtr){
				// 追加
				memory.storeSkipPointer(index + 1, type, term.ptr, false, MEM_SKIPPTR);
			}
		}
	}

	private void makeSkipPtrForContent(ContentIDs term, int type){
		int index = term.skipPtrIndex;

		if(index == -1){
			// 最初の部分書き換え
			memory.storeSkipPointer(0, type, term.currentPtr, true, MEM_SKIPCONT);
			refreshSkipPtrContent(type);
		}else{
			if(term.addSkipPtr){
				// 追加
				memory.storeSkipPointer(index + 1, type, term.currentPtr, false, MEM_SKIPCONT);
			}
		}
	}

	// 1番目のスキップポインタに固まらない対策
	private void refreshSkipPtrDict(){
		LookupPointer threshold = memory.lookupSkipPointer(1);
		LookupPointer nextPtr = memory.lookupSkipPointer(0);
		LookupPointer prevPtr = null;
		int count = 0;
		if(threshold == null){
			while(nextPtr != null){
				prevPtr = nextPtr;
				nextPtr = memory.lookupDictionaryNext(nextPtr);
				count++;
			}
		}else{
			while(!threshold.equals(nextPtr)){
				prevPtr = nextPtr;
				nextPtr = memory.lookupDictionaryNext(nextPtr);
				count++;
			}
		}

		if(count >= DIGLENGTH){
			memory.storeSkipPointer(1, 0, prevPtr, false, MEM_SKIPPTR);
		}
	}

	// 1番目のスキップポインタに固まらない対策
	private void refreshSkipPtrContent(int type){
		LookupPointer threshold = memory.lookupContentsSkipPointer(1, type);
		LookupPointer nextPtr = memory.lookupContentsSkipPointer(0, type);
		LookupPointer prevPtr = null;
		int count = 0;
		if(threshold == null){
			while(nextPtr != null){
				prevPtr = nextPtr;
				nextPtr = memory.lookupNextContentPointer(nextPtr);
				count++;
			}
		}else{
			while(nextPtr != null && !threshold.equals(nextPtr)){
				prevPtr = nextPtr;
				nextPtr = memory.lookupNextContentPointer(nextPtr);
				count++;
			}
		}

		if(count >= DIGLENGTH){
			// System.out.println("Count: " + count);
			memory.storeSkipPointer(1, type, prevPtr, false, MEM_SKIPCONT);
		}
	}

	// Store an term
	private ExtendTerm storeDictionary(String query, int valueType) throws UnsupportedEncodingException{
		ExtendTerm newTerm = lookupTerm(query, valueType);

		if(newTerm.ptr == null){
			// System.out.println("query " + query + " cannot found.");
			// Add term if a query doesn't appear
			LookupPointer newTermPtr = memory.storeDictionary(query, newTerm.next);
			newTerm.ptr = newTermPtr;

			// Reconnect pointer to the term
			if(newTerm.prev != null) memory.storeDictionaryPointer(newTerm.prev, newTermPtr);

			// Add and modify dictionary
			makeSkipPtrForTerm(newTerm, MEM_SKIPPTR);

			this.termcounter++;
		}else memory.incrementDocFreq(newTerm.ptr);

		return newTerm;
	}

	// Get contents ID and term frequency by using pointer to invert index
	public LinkedHashMap<Long, Integer> getContentsByHash(LookupPointer ptr, int length){
		LinkedHashMap<Long, Integer> indexMap = new LinkedHashMap<Long, Integer>(length);
		LookupPointer currentPtr = ptr;
		LookupPointer lastPtr = null;
		InvertIndexData hash;
		long firstID = ptr.firstID;
		long lastDiff = 0;
		long tmpHash = 0;

		if(ptr.equals(LookupPointer.EMPTY) || firstID == -1) return null;

		for(int i = 0; i < length && currentPtr != null; i++){
			hash = memory.lookupInvertIndex(currentPtr);
			tmpHash = firstID - lastDiff;
			lastDiff += hash.getID();
			// System.out.println("Add " + tmpHash);
			indexMap.put(tmpHash, hash.getTermfreq());

			lastPtr = currentPtr;
			currentPtr = memory.lookupNextInvIndex(lastPtr);
		}

		// <!> ptrの中身を破壊的に変更
		if(currentPtr != null) ptr.modify(lastPtr.block, lastPtr.point, tmpHash);
		else ptr.modify(-1, -1, -1);

		return indexMap;
	}

	private void storeInvIndex(LookupPointer dicPtr, long ID, int termfreq, int valueType){
		// 転置インデックスのポインタ取得
		LookupPointer prevPtr = memory.lookupDictionaryInvIndex(dicPtr, valueType);

		// ID差分計算
		long firstID = memory.lookupDictionaryFirstID(dicPtr, valueType);
		long diff;
		if(firstID == Long.MIN_VALUE) diff = 0L;
		else diff = ID - firstID;

		// 転置インデックス書き込み
		LookupPointer invIndexPtr = memory.storeInvertIndex(prevPtr, diff, termfreq);

		// 辞書ポインタ更新
		memory.storeDictionaryToInvIndex(dicPtr, invIndexPtr, ID, valueType);
	}

	// Look up pointers specified by ID
	private ContentIDs lookupContentPointer(long ID, int valueType){
		searchcounter++;
		// look up skip pointer
		int skipPointerLength = memory.getSkipContentsLength(valueType);
		long nextID = 0;
		LookupPointer nextPtr = null;
		LookupPointer prevPtr = null;
		int i = 0;
		int skipLevel = 0;
		int skipVal = 1;
		if(skipPointerLength > 1){
			skipLevel = (int)(Math.log(skipPointerLength - 1) / Math.log(DIGLENGTH));
			skipVal = (int)Math.pow(DIGLENGTH, skipLevel);
		}
		for(i = 0; i < skipPointerLength; i += skipVal){
			nextPtr = memory.lookupContentsSkipPointer(i, valueType);
			// No pointer
			if(nextPtr == null) break;

			nextID = memory.lookupContentID(nextPtr);
			if(nextID > ID){
				// before this ID
				skipLevel--;
				if(skipLevel == -1 || i == 0){
					i--;
					break;
				}
				i -= skipVal;
				skipVal = (int)Math.pow(DIGLENGTH, skipLevel);
			}else if(nextID == ID){
				// Just this content
				return new ContentIDs(ID, nextPtr, null, null, false, i);
			}

			// look at past
			if(i + skipVal >= skipPointerLength){
				do{
					skipLevel--;
					if(skipLevel == -1){
						skipVal = 1;
						break;
					}
					skipVal = (int)Math.pow(DIGLENGTH, skipLevel);
				}while(i + skipVal >= skipPointerLength);
			}
		}

		// the first content
		if(i == -1) return new ContentIDs(ID, null, null, memory.lookupContentsSkipPointer(0, valueType),
				true, -1);
		else{
			if(i == skipPointerLength) i--;
			nextPtr = memory.lookupContentsSkipPointer(i, valueType);
		}

		// trace contents by ID
		int count = 0;
		prevPtr = nextPtr;
		while(nextPtr != null){
			long l = memory.lookupContentID(nextPtr);
			if(l > ID){
				// nextTermが上 -> 辞書になかった
				return new ContentIDs(-1, null, prevPtr, nextPtr, (count >= DIGLENGTH), i);
			}else if(l == ID){
				// ID matched -> nextPtr is actual existing point
				return new ContentIDs(l, nextPtr, prevPtr, null, false, i);
			}else{
				prevPtr = nextPtr;
				nextPtr = memory.lookupNextContentPointer(nextPtr);
			}
			count++;
		}
		return new ContentIDs(ID, null, prevPtr, nextPtr, (count >= DIGLENGTH), i);
	}

	// Method for storing content
	public void storeContent(String content, long ID, int index, LookupPointer[] appendix){
		ContentIDs contTerm = lookupContentPointer(ID, index);

		// IDがかぶったということは→ 書き換えないで破棄
		if(contTerm.currentPtr == null){
			try{
				contentLock.lock();
				LookupPointer storePtr = memory.storeContents(content, ID, index, contTerm.nextPtr, appendix);
				contTerm.currentPtr = storePtr;

				// 前からポインタ接続
				if(contTerm.prevPtr != null) memory.storeContentNext(contTerm.prevPtr, storePtr);
			}finally{
				contentLock.unlock();
			}
			// スキップポインタ追加
			makeSkipPtrForContent(contTerm, index);
		}
		indexcounter++;
	}

	// Store appendix on memory
	public LookupPointer storeAppendix(String content, long ID, int index){
		// If any content with same ID is found, it does not over write
		return memory.storeAppendix(content, index);
	}

	public void deleteContent(long ID){
		for(int i = 0; i < contentInfo.getContentNum(); i++){
			ContentIDs infos = lookupContentPointer(ID, i);
			if(infos.currentPtr != null){
				memory.deleteContents(infos.currentPtr);
				indexcounter--;
			}
		}
	}

	// backup ///////////////
	public void revertLatest(){
		memory.revert(-1);
	}

	public void revertByEntry(long ID){
		memory.revert(ID);
	}

	public long takeSnapshot(){
		return memory.snapshot();
	}

	public void backupLatest(){
		memory.snapshot();
		memory.backup();
	}
	
	public long latestBackupIndex() {
		return memory.latestBackupIndex();
	}

	// ///////////////////////

	public void synchronizeClock(){
		memory.synchronizeClock();
	}

	public void optimizeMemory(){
		memory.optimizeMemory();
	}

	public void showBackupList(){
		memory.showBackupList();
	}
}
